{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-01-06T12:34:45.534586Z",
     "start_time": "2025-01-06T12:34:39.845885Z"
    }
   },
   "source": [
    "import torch\n",
    "from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n",
    "from datasets import load_dataset\n",
    "\n",
    "\n",
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32\n",
    "\n",
    "model_id = \"openai/whisper-large-v3-turbo\"\n",
    "\n",
    "print(\"Loading model...\")\n",
    "model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True,cache_dir=\"data-cache\")\n",
    "\n",
    "model.generation_config.language = \"en\"  # for hindi\n",
    "model.to(device)\n",
    "print(\"Model loaded.\")\n",
    "processor = AutoProcessor.from_pretrained(model_id)\n",
    "\n",
    "print(\"Loading dataset...\")\n",
    "pipe = pipeline(\n",
    "    \"automatic-speech-recognition\",\n",
    "    model=model,\n",
    "    tokenizer=processor.tokenizer,\n",
    "    feature_extractor=processor.feature_extractor,\n",
    "    torch_dtype=torch_dtype,\n",
    "    device=device,\n",
    ")\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading model...\n",
      "Model loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "ffd67fe81c2b85df"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-06T12:34:52.895497Z",
     "start_time": "2025-01-06T12:34:49.399637Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 把音频路径，加载成numpy格式数据\n",
    "from scipy.io import wavfile\n",
    "\n",
    "# 加载音频文件\n",
    "audio_file = \"/Users/luckincoffee/Desktop/project/python/aigc-python/english.wav\"\n",
    "sampling_rate, data = wavfile.read(audio_file)\n",
    "print(f\"音频数据: {data}\")\n",
    "print(f\"采样率: {sampling_rate}\")\n",
    "result = pipe(data)\n",
    "\n",
    "print(result[\"text\"])\n"
   ],
   "id": "3be9adb254ae20a5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "音频数据: [ 0 -1  1 ...  0 -1  1]\n",
      "采样率: 44100\n",
      " Who?\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-06T12:53:42.610142Z",
     "start_time": "2025-01-06T12:53:17.625701Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from transformers import AutoProcessor, SeamlessM4Tv2Model\n",
    "import torchaudio\n",
    "\n",
    "processor = AutoProcessor.from_pretrained(\"facebook/seamless-m4t-v2-large\")\n",
    "model = SeamlessM4Tv2Model.from_pretrained(\"facebook/seamless-m4t-v2-large\",cache_dir=\"data-cache\")\n",
    "\n",
    "# from text\n",
    "# text_inputs = processor(text = \"Hello, my dog is cute\", src_lang=\"eng\", return_tensors=\"pt\")\n",
    "# audio_array_from_text = model.generate(**text_inputs, tgt_lang=\"rus\")[0].cpu().numpy().squeeze()\n",
    "\n",
    "# from audio\n",
    "audio, orig_freq =  torchaudio.load(\"/Users/luckincoffee/Desktop/project/python/aigc-python/english.wav\")\n",
    "audio =  torchaudio.functional.resample(audio, orig_freq=orig_freq, new_freq=16_000) # must be a 16 kHz waveform array\n",
    "audio_inputs = processor(audios=audio, return_tensors=\"pt\")\n",
    "audio_array_from_audio = model.generate(**audio_inputs, tgt_lang=\"rus\")[0].cpu().numpy().squeeze()\n"
   ],
   "id": "13859efba639703f",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations\n",
      "  warnings.warn(\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "preprocessor_config.json:   0%|          | 0.00/1.78k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "e1746fb5c6f1477ca25e0f6806295fc6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/19.7k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "e3aeb307e6864db4a53ebaf9de328ebf"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "sentencepiece.bpe.model:   0%|          | 0.00/5.17M [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "3859799ae60f4ce389e038bc647c3622"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "added_tokens.json:   0%|          | 0.00/2.07k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "05ca4875c64f4e029316ecf018db6e34"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/2.34k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "4d210456f6b44537a7c45ddd209fe34a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "config.json:   0%|          | 0.00/2.72k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "dbb62aebde34455088cce33308f365ab"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "model.safetensors.index.json:   0%|          | 0.00/211k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "57c5e21428a44660be22eb1ef1951da2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "84127f9cc159447283fc1a017fb0acb1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "model-00001-of-00002.safetensors:   0%|          | 0.00/5.00G [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "5642c23fe46145dda24e7264497ea717"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "c9b99a0d17cd5239"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-23T09:42:14.240028Z",
     "start_time": "2025-01-23T09:40:49.843841Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from optimum.onnxruntime  import ORTModelForSpeechSeq2Seq\n",
    "import torch\n",
    "from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n",
    "\n",
    "\n",
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32\n",
    "\n",
    "model_id = 'openai/whisper-large-v3-turbo'\n",
    "\n",
    "model = ORTModelForSpeechSeq2Seq.from_pretrained(model_id,\n",
    "                                                 export=True,\n",
    "                                                 cache_dir='data-cache')\n",
    "\n",
    "model.save_pretrained(model_id)\n",
    "processor = AutoProcessor.from_pretrained(model_id)\n",
    "\n",
    "print('Loading dataset...')\n",
    "pipe = pipeline(\n",
    "    'automatic-speech-recognition',\n",
    "    model=model,\n",
    "    tokenizer=processor.tokenizer,\n",
    "    feature_extractor=processor.feature_extractor,\n",
    "    torch_dtype=torch_dtype,\n",
    "    device=device,\n",
    ")\n",
    "\n",
    "def get_audio_data(audio_path: str) -> str:\n",
    "    print('Getting audio data...'+audio_path)\n",
    "    result =pipe(audio_path, generate_kwargs={'language': 'chinese'})\n",
    "    print('Audio data got.')\n",
    "    print(result)\n",
    "    return result['text']\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print(get_audio_data('/Users/luckincoffee/Desktop/project/python/aigc-python/english.mp3'))\n"
   ],
   "id": "5765090617b6a6d1",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
      "Non-default generation parameters: {'begin_suppress_tokens': [220, 50256]}\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:1165: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if input_features.shape[-1] != expected_seq_length:\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:344: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:383: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if input_shape[-1] > 1 or self.sliding_window is not None:\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if past_key_values_length > 0:\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:351: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  if attention_mask.size() != (bsz, 1, tgt_len, src_len):\n",
      "/opt/anaconda3/envs/llm/lib/python3.10/site-packages/transformers/models/whisper/modeling_whisper.py:306: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  and past_key_value[0].shape[2] == key_value_states.shape[1]\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n",
      "Non-default generation parameters: {'begin_suppress_tokens': [220, 50256]}\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n",
      "Getting audio data.../Users/luckincoffee/Desktop/project/python/aigc-python/english.mp3\n",
      "Audio data got.\n",
      "{'text': '1 2 3'}\n",
      "1 2 3\n"
     ]
    }
   ],
   "execution_count": 1
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
